package parser

import(
	"os"
	"fmt"
	"io/ioutil"
	"log"
	"regexp"
	"strings"
	"search_backend"
)

func splitContent (content string, separator string) []string {
	words := strings.Split (content, separator)
	return words;
}

func inArray (term string, array [] string) bool {
	for _, arrayTerm := range (array) {
		if (arrayTerm == term) {
			return true;
		}
	}
	return false;
}

func isNumber (term string) bool {
	matched,_  := regexp.MatchString("[0-9]+", term)
	return matched || isHexNumber (term);
}

func isHexNumber (term string) bool {
	matched,_ := regexp.MatchString("0x[0-9a-fA-F]+", term)
	return matched
}

func isLongEnough (term string) bool {
	return len(term) > 2
}

func ParseFile (docPath string, stoplist []string) []string {
	content, err := ioutil.ReadFile (docPath);
	if (err != nil) {
		log.Fatal (err)
		return nil
	}

	log.Printf("Parsing file \"%s\"", docPath)
	var words []string;
	pattern := regexp.MustCompile(`[a-zA-Z0123456789ñáéíóúçàèìòùâêîôûºª]+`);
	tokens := pattern.FindAll (content, -1);
	if (tokens != nil) {
		for _, term := range (tokens) {
			stringTerm := string (term)
			if (isLongEnough (stringTerm) && !isNumber (stringTerm) && !inArray (stringTerm, stoplist)) {
				words = append (words, strings.ToLower(string(term)))
			}
		}
	}

	/* Check for emails, ip's, urls and MACs*/
	pattern = regexp.MustCompile(`[a-zA-Z0-9\.\_\%\-]+\@[a-zA-Z0-9\.\-]+\.[a-zA-Z][a-zA-Z][a-zA-Z]?[a-zA-Z]?|[0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?\.[0-9][0-9]?[0-9]?|[0-9a-fA-F][0-9a-fA-F]?\:[0-9a-fA-F][0-9a-fA-F]?\:[0-9a-fA-F][0-9a-fA-F]?\:[0-9a-fA-F][0-9a-fA-F]?\:[0-9a-fA-F][0-9a-fA-F]?\:[0-9a-fA-F][0-9a-fA-F]|(http\:\/\/www\.|ftp\:\/\/|www\.|http\:\/\/)[a-zA-Z0-9\.\-]+\.[a-zA-Z0-9][a-zA-Z0-9][a-zA-Z0-9]?[a-zA-Z0-9]?`);
	tokens = pattern.FindAll (content, -1);
	if (tokens != nil) {
		for _, term := range (tokens) {
			words = append (words, strings.ToLower(string(term)))
		}
	}

	return words
}

// Full directory parsing
func ParseFiles(dirname string) {
    d, err := os.Open(dirname)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    fi, err := d.Readdir(-1)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

		// Parse the stoplist
		stoplist := ParseFile("res/stoplist.txt", nil);

    for _, fi := range fi {
			if (fi.IsRegular() && fi.Name[0] != '_') {
				path := fmt.Sprintf("%s/%s", dirname, fi.Name)
				words := ParseFile(path, stoplist)
				search_backend.AddDocument(path)
				for _, word := range words {
					search_backend.AddWord(path, word)
				}
			}
    }
}
